ser@ardbc01 ~]$ less stagingstandby/copy.forhotstandby.legacy.sh 
#!/bin/sh
############################################################################
# $Author: cbrothers $
# $Date: 2015/12/15 11:16:34 $
# $Revision: 1.6 $
#
############################################################################
# Copyright (C) 2007 Dominion Enterprises All Rights Reserved
############################################################################
##  This script is invoked by the ARCHIVE_COMMAND parameter in the postgres
##  config file. It is run on server PROD_LEGACY_PSQL.SERVER.INT.
##         It will do the following:
##   
##   1. Copy the informational .backup and .history files to staging library
##   2. Gzip the actual xlogs to the staging library
##   3. Ping DR-LEGACY-PSQL.SERVER.INT to test for network connectivity
##   4. scp the following to DR-legacy-PSQL.SERVER.INT
##       -   informational log files 
##       -   the straight unzipped pg_log files for use in hot standby
##
##   The staging library on PROD-LEGACY-PSQL.SERVER.INT is:
##       /mnt/dbdumps/postgres-legacy/archlog
##
##  Copied to CVS FROM folder /MavMail/mm/InternalTools
##                                 /database_replication/ar-db-2desq
##
##  MES- 3/18/2015: if first testping successful, failover after 10 minutes
##     of unsuccessful scp
###########################################################################

REQ_FILE=$1
DEST=$2
GZDEST="${DEST}.gz"
LF="${REQ_FILE}.lock"
SUFFIX=${REQ_FILE##*.}
DATETIME=`date +"%Y%m%d-%H%M%S"`
PING='/bin/ping -c 1 -W 1'
##/usr/bin/lockfile "${LF}"  # lockfile also on receiving  server


#####################################################################
## logic to gzip xlogs to staging
####################################################################
if [ "${SUFFIX}" != 'history' ] && [ "${SUFFIX}" != 'backup' ]; then
  /bin/cat "${REQ_FILE}" | gzip > /mnt/dbdumps/postgres-legacy/archlog/${GZDEST}
  if [ "$?" -gt "0" ] ; then
    echo 'unsuccessful gzip of archlog file' >> /tmp/archlog.legacy.log
    echo 'the unsuccessful gzip exit code is '  "$?" >> /tmp/archlog.legacy.log
  fi
else
#####################################################################
## logic to copy informational files to staging
####################################################################
   cp "${REQ_FILE}"  "/mnt/dbdumps/postgres-legacy/archlog/${DEST}"
   if [ "$?" -gt "0" ] ; then
      echo " unsuccessful unzipped copy of backup info file" >> /tmp/archlog.legacy.log
      echo 'the unsuccessful backup file copy is ' "$?"  >> /tmp/archlog.legacy.log
   fi
fi
badping=0
${PING} 10.248.156.47
##########################################################################
## upon successful ping of dr-legacy-psql.server.int, scp to /shared/ardbc-dc3/archlog-legacy
##    folder on that server
##
##  Postgres should now will handle outages up to 30 minutes
##   if greater than 30 minutes dba should be called
#####################################################################
## 01/15/2015 - logic to scp current log directly to dr server - for hotstandby
##########################################################################
if [ "$?" = "0" ] ; then
 echo "the datetime before the scp to /shared/ardbc-dc3/archlog-legacy is " >> /tmp/archlog.legacy.log
  /bin/date  >> /tmp/archlog.legacy.log

  scp "${REQ_FILE}"  "pguser@dr-legacy-psql.server.int:/shared/ardbc-dc3/archlog-legacy/${DEST}"
 
  if [ "$?" != "0" ] ;  then
     echo "first log scp to dr-legacy-psql.server.int not good - trying again after sleeping 5 minutes. time is ${DATETIME}  " >> /tmp/archlog.legacy.log
     sleep 300
       scp "${REQ_FILE}"  "pguser@dr-legacy-psql.server.int:/shared/ardbc-dc3/archlog-legacy/${DEST}"
       if [ "$?" != "0" ] ;  then
         echo "second log scp to dr-legacy-psql.server.int not good - trying again after sleeping another 5 minutes. time is ${DATETIME} " >> /tmp/archlog.legacy.log
         sleep 300
         scp "${REQ_FILE}"  "pguser@dr-legacy-psql.server.int:/shared/ardbc-dc3/archlog-legacy/${DEST}"
         if  [ "$?" != "0" ] ;  then
           echo " third attempt at log scp to dr-legacy-psql.server.int failed - calling dba.  time is ${DATETIME}. log is ${DEST}  " >> /tmp/archlog.legacy.log 
           /MavMail/mm/InternalTools/system_tools/send_mail.pl 'markcell@autorevenue.com,ddsdba_autoalert@drivedominion.com' 'dbalert@prod-legacy-psql.server.int' 'dr-legacy-psql.server.int log scp unsuccessful after 3 attempts.' 'call m
ark 413-433-3124'
         fi
     fi 
  fi
 echo " the datetime after the scp to legacy /shared/ardbc-dc3/archlog-legacy folder is " >> /tmp/archlog.legacy.log
  /bin/date >> /tmp/archlog.legacy.log
else
  number=0
  badping=1
  echo "unsuccessful ping of dr-legacy-psql.server.int - first time - sleeping for 1 minute.  time is ${DATETIME}   " >> /tmp/archlog.legacy.log
  while [ "$badping" = '1' ] ; do
     echo "copying current archive log ${GZDEST} to holdinglegacy directory.      time is ${DATETIME} " >> /tmp/archlog.legacy.log
     /bin/cp -u  "/mnt/dbdumps/postgres-legacy/archlog/${GZDEST}" "/home/pguser/holdinglegacy/${GZDEST}"
     sleep 60
     number=$((number + 60))
     ${PING} 10.248.156.47
     if  [ "$?" = "0" ]   ;
     then
        echo " successful ping of dr-legacy-psql.server.int after $number seconds: sleeping for 20 seconds then continuing with scp copy from holdinglegacy. time is ${DATETIME} " >> /tmp/archlog.legacy.log
        sleep 20
        badping=0
        echo "the datetime before the scp to (dr-legacy)/shared/ardbc-dc3/archlog-legacy from the staging library is " >> /tmp/archlog.legacy.log
         /bin/date >> /tmp/archlog.legacy.log
         cd /home/pguser/holdinglegacy/
         gunzip *
         scp *  pguser@dr-legacy-psql.server.int:/shared/ardbc-dc3/archlog-legacy/ >> /tmp/archlog.legacy.log
         if  [ "$?" = "0" ]   ;
         then
             echo "the datetime after the successful scp from the staging library is " >> /tmp/archlog.legacy.log
             /bin/date >> /tmp/archlog.legacy.log
             /bin/rm /home/pguser/holdinglegacy/* -f
         else
             echo "unsuccessful scp to dr legacy after successful ping - sleeping for another minute " >> /tmp/archlog.legacy.log
         fi
     else
        echo " unsuccessful ping of prod-legacy-psql.server.int after $number seconds - sleeping for another minute. time is ${DATETIME} " >> /tmp/archlog.legacy.log
     fi
     if [ "$number" -ge "3600" ] ; then
        echo "  ping has been unsuccessful for more than 60 minutes - calling dba. time is ${DATETIME} " >> /tmp/archlog.legacy.log
            /MavMail/mm/InternalTools/system_tools/send_mail.pl 'markcell@autorevenue.com,ddsdba_autoalert@drivedominion.com' 'dbalert@prod-legacy-psql.server.int' 'prod-legacy-psql.server.int ping unsuccessful after 30 minutes'  'call mark 413-433-3124'
    fi
  done
fi
# 

sleep 60

badping=0
badscp=0
echo "this is the echo before the ping to arprodpsqlreplication01 " >> /tmp/archlog.legacy.toreplication.log

${PING} 10.93.156.52
##########################################################################
## upon successful ping of arprodpsqlreplication01, scp to 
##    /database/logship/PSQL_94/archlog-legacy
##
##  Postgres should now will handle outages up to 30 minutes
##   if greater than 30 minutes dba should be called
#####################################################################
## 01/15/2015 - logic to scp current log directly to dr server - for hotstandby
##########################################################################
if [ "$?" = "0" ] ; then
 echo "the datetime before the scp to /database/logship/PSQL_94/archlog-legacy is " >> /tmp/archlog.legacy.toreplication.log
  /bin/date  >> /tmp/archlog.legacy.toreplication.log

echo "this is the echo after the check of the ping to arprodpsqlreplication01 " >> /tmp/archlog.legacy.toreplication.log

  scp "${REQ_FILE}"  "pguser@arprodpsqlreplication01:/database/logship/PSQL_94/archlog-legacy/${DEST}"

if [ "$?" != "0" ] ;  then
     echo "the return code for first unsuccessful scp is  "$?" time is  ${DATETIME}  " >> /tmp/archlog.legacy.toreplication.log
     echo "first log scp to arprodpsqlreplication01 not good - trying again after sleeping 20 minutes. time is ${DATETIME}  " >> /tmp/archlog.legacy.toreplication.log
     sleep 1200
     badscp=1
  else
      echo " the log just copied is ${DEST}  " >> /tmp/archlog.legacy.toreplication.log
      echo " the datetime after the scp to arprodpsqlreplication01 legacy /database/logship/PSQL_94/archlog-legacy folder is   " >> /tmp/archlog.legacy.toreplication.log
      /bin/date >> /tmp/archlog.legacy.toreplication.log
     badscp=0
     exit 0
  fi

    scp "${REQ_FILE}"  "pguser@arprodpsqlreplication01:/database/logship/PSQL_94/archlog-legacy/${DEST}"
    if [ "$?" != "0" ] ;  then
       echo "the return code for second unsuccessful scp is  "$?" time is  ${DATETIME}  " >> /tmp/archlog.legacy.toreplication.log
       echo "second log scp to arprodpsqlreplication01 not good - trying again after sleeping another 20 minutes. time is ${DATETIME} " >> /tmp/archlog.legacy.toreplication.log
       sleep 1200
       badscp=1
    else
         echo " the log just copied is ${DEST}  " >> /tmp/archlog.legacy.toreplication.log
         echo " the datetime after the scp to arprodpsqlreplication01 legacy /database/logship/PSQL_94/archlog-legacy folder is " >> /tmp/archlog.legacy.toreplication.log
        /bin/date >> /tmp/archlog.legacy.toreplication.log
       badscp=0
       exit 0
    fi

    scp "${REQ_FILE}"  "pguser@arprodpsqlreplication01:/database/logship/PSQL_94/archlog-legacy/${DEST}"
     if  [ "$?" != "0" ] ;  then
         touch /home/pguser/legacy.logship.failure
        echo " third attempt at log scp to arprodpsqlreplication01 failed - calling dba.  time is ${DATETIME}. log is ${DEST}  " >> /tmp/archlog.legacy.toreplication.log
        /MavMail/mm/InternalTools/system_tools/send_mail.pl 'markcell@autorevenue.com,ddsdba_autoalert@drivedominion.com' 'dbalert@prod-legacy-psql.server.int' 'arprodpsqlreplication01 log scp unsuccessful after 3 attempts.' 'call mark 413-433-3124'
     else
       echo " the datetime after the scp to arprodpsqlreplication01 legacy /database/logship/PSQL_94/archlog-legacy folder is " >> /tmp/archlog.legacy.toreplication.log
       /bin/date >> /tmp/archlog.legacy.toreplication.log
       badscp=0
     fi
 
echo " the log just copied is ${DEST}  " >> /tmp/archlog.legacy.toreplication.log
echo " the datetime after the scp to arprodpsqlreplication01 legacy /database/logship/PSQL_94/archlog-legacy folder is " >> /tmp/archlog.legacy.toreplication.log
       /bin/date >> /tmp/archlog.legacy.toreplication.log

else
  number=0
  badping=1
  echo "unsuccessful ping of arprodpsqlreplication01 - first time - sleeping for 1 minute.  time is ${DATETIME}   " >> /tmp/archlog.legacy.toreplication.log
  while [ "$badping" = '1' ] ; do
     echo "copying current archive log ${GZDEST} to holdinglegacy directory.      time is ${DATETIME} " >> /tmp/archlog.legacy.toreplication.log
     /bin/cp -u  "/mnt/dbdumps/postgres-legacy/archlog/${GZDEST}" "/home/pguser/holdingreplegacy/${GZDEST}"
     sleep 60
     number=$((number + 60))
     ${PING} 10.93.156.52
     if  [ "$?" = "0" ]   ;
     then
        echo " successful ping of arprodpsqlreplication01 after $number seconds: sleeping for 20 seconds then continuing with scp copy from holdingreplegacy. time is ${DATETIME} " >> /tmp/archlog.legacy.toreplication.log
        sleep 20
        badping=0
        echo "the datetime before the scp to (arprodpsqlreplication01)/database/logship/PSQL_94/archlog-legacy from the staging library is " >> /tmp/archlog.legacy.toreplication.log
         /bin/date >> /tmp/archlog.legacy.toreplication.log
         cd /home/pguser/holdingreplegacy/
         gunzip *
         scp *  pguser@arprodpsqlreplication01:/database/logship/PSQL_94/archlog-legacy/ >> /tmp/archlog.legacy.toreplication.log
         if  [ "$?" = "0" ]   ;
         then
             echo "the datetime after the successful scp from the staging library is " >> /tmp/archlog.legacy.toreplication.log
             /bin/date >> /tmp/archlog.legacy.toreplication.log
             /bin/rm /home/pguser/holdingreplegacy/* -f
         else
             echo "unsuccessful scp to arprodpsqlrepl01 legacy after successful ping - sleeping for another minute " >> /tmp/archlog.legacy.toreplication.log
         fi
     else
        echo " unsuccessful ping of arprodpsqlreplication01 after $number seconds - sleeping for another minute. time is ${DATETIME} " >> /tmp/archlog.legacy.toreplication.log
     fi
     if [ "$number" -ge "3600" ] ; then
        echo "  ping has been unsuccessful for more than 60 minutes - calling dba. time is ${DATETIME} " >> /tmp/archlog.legacy.toreplication.log
            /MavMail/mm/InternalTools/system_tools/send_mail.pl 'markcell@autorevenue.com,ddsdba_autoalert@drivedominion.com' 'dbalert@prod-legacy-psql.server.int' 'arprodpsqlreplication01  ping unsuccessful after 30 minutes'  'call mark 413-433-3124'
    fi
  done
fi
#